package ctrip.demo

import org.apache.spark.sql.types.{StringType, StructField, StructType}
import org.apache.spark.sql.{Row, SQLContext}
import org.apache.spark.{SparkConf, SparkContext}

import scala.util.control.Breaks

/**
 * 暂无用到
 * Created by gongenbo on 2017/5/15.
 */
object CtripOrderForecast {
  var limit: Int = 1000
  def main(args: Array[String]): Unit = {

    val sparkConf = new SparkConf().setAppName("CtripOrderForecast").setMaster("local").set("spark.streaming.unpersist", "true")
      .set("spark.shuffle.consolidateFiles", "true").set("spark.default.parallelism", "1").set("spark.sql.shuffle.partitions", "1");

    val sc = new SparkContext(sparkConf)
    val sqlContext = new SQLContext(sc)
    //加载训练数据
    /*
    val df = sqlContext.load("com.databricks.spark.csv",
      Map("path"->"F:\\Study\\ml\\match\\ctrip\\ctrip\\competition_train.txt",
        "header"->"true","delimiter"->"\t"))*/
    //val lines=sc.textFile("/data/ctrip/competition_train.txt", 1)
    val lines=sc.textFile("F:\\Study\\ml\\match\\ctrip\\ctrip\\competition_train.txt", 1)
    //val rdd=lines.map(_.split("\t"))

    val rowRdd=lines.filter(lines=>{
        var flag:Boolean=true
        var i=0
        var len=lines.length
        val line=lines.split("\t")
        val loop = new Breaks;
        loop.breakable {
          while(i<157){
            //过滤掉第一行字段名
            //过滤除了user_confirmtime以外所有值为负数的数据
            if((line(i).toString.startsWith("-")&&i!=27)/*||line(0).equals("orderid")*/){
              flag=false
              loop.break
            }
            i=i+1
          }
        }
        flag
    }).map(lines=>{
      val line=lines.split("\t")
      Row(line(0),line(1),line(2),line(3),line(4),line(5),line(6),line(7),line(8),line(9),line(10),line(11),line(12),line(13),line(14),line(15),line(16),line(17),line(18),line(19),line(20),line(21),line(22),line(23),line(24),line(25),line(26),line(27),line(28),line(29),line(30),line(31),line(32),line(33),line(34),line(35),line(36),line(37),line(38),line(39),line(40),line(41),line(42),line(43),line(44),line(45),line(46),line(47),line(48),line(49),line(50),line(51),line(52),line(53),line(54),line(55),line(56),line(57),line(58),line(59),line(60),line(61),line(62),line(63),line(64),line(65),line(66),line(67),line(68),line(69),line(70),line(71),line(72),line(73),line(74),line(75),line(76),line(77),line(78),line(79),line(80),line(81),line(82),line(83),line(84),line(85),line(86),line(87),line(88),line(89),line(90),line(91),line(92),line(93),line(94),line(95),line(96),line(97),line(98),line(99),line(100),line(101),line(102),line(103),line(104),line(105),line(106),line(107),line(108),line(109),line(110),line(111),line(112),line(113),line(114),line(115),line(116),line(117),line(118),line(119),line(120),line(121),line(122),line(123),line(124),line(125),line(126),line(127),line(128),line(129),line(130),line(131),line(132),line(133),line(134),line(135),line(136),line(137),line(138),line(139),line(140),line(141),line(142),line(143),line(144),line(145),line(146),line(147),line(148),line(149),line(150),line(151),line(152),line(153),line(154),line(155),line(156))})
    println("rowRdd count:"+rowRdd.count())
    //val schemaString=lines.take(1).map(_.split("\t"))
    //val schemaString = sc.textFile("/data/ctrip/competition_train.txt", 1).take(1).toString;
    //创建schema
    //val schema = StructType(schemaString(0).split("\t").map(fieldName => StructField(fieldName, StringType, true)))

    val schemaString="orderid\tuid\torderdate\thotelid\tbasicroomid\troomid\torderlabel\tstar\trank\treturnvalue\tprice_deduct\tbasic_minarea\tbasic_maxarea\troomservice_1\troomservice_2\troomservice_3\troomservice_4\troomservice_5\troomservice_6\troomservice_7\troomservice_8\troomtag_1\troomtag_2\troomtag_3\troomtag_4\troomtag_5\troomtag_6\tuser_confirmtime\tuser_avgadvanceddate\tuser_avgstar\tuser_avggoldstar\tuser_avgrecommendlevel\tuser_avgroomnum\tordertype_1_ratio\tordertype_2_ratio\tordertype_3_ratio\tordertype_4_ratio\tordertype_5_ratio\tordertype_6_ratio\tordertype_7_ratio\tordertype_8_ratio\tordertype_9_ratio\tordertype_10_ratio\tordertype_11_ratio\tuser_avgdealpriceholiday\tuser_avgdealpriceworkday\tuser_avgdealprice\tuser_avgpromotion\tuser_avgprice_star\torderbehavior_1_ratio\torderbehavior_2_ratio\torderbehavior_3_ratio_1week\torderbehavior_4_ratio_1week\torderbehavior_5_ratio_1week\torderbehavior_3_ratio_1month\torderbehavior_4_ratio_1month\torderbehavior_5_ratio_1month\torderbehavior_3_ratio_3month\torderbehavior_4_ratio_3month\torderbehavior_5_ratio_3month\torderbehavior_6_ratio\torderbehavior_7_ratio\torderbehavior_8\torderbehavior_9\tuser_ordernum\tuser_activation\tuser_avgprice\tuser_maxprice\tuser_minprice\tuser_stdprice\tuser_cvprice\tuser_citynum\tuser_avgroomarea\tuser_roomservice_4_0ratio\tuser_roomservice_4_2ratio\tuser_roomservice_4_3ratio\tuser_roomservice_4_4ratio\tuser_roomservice_4_1ratio\tuser_roomservice_4_5ratio\tuser_roomservice_3_123ratio\tuser_roomservice_6_2ratio\tuser_roomservice_6_1ratio\tuser_roomservice_6_0ratio\tuser_roomservice_5_1ratio\tuser_roomservice_7_0ratio\tuser_roomservice_2_1ratio\tuser_roomservice_8_1ratio\tuser_rank_ratio\tuser_roomservice_5_345ratio\tuser_ordnum_1week\tuser_avgprice_1week\tuser_medprice_1week\tuser_minprice_1week\tuser_maxprice_1week\tuser_roomservice_3_123ratio_1week\tuser_roomservice_7_1ratio_1week\tuser_roomservice_7_0ratio_1week\tuser_roomservice_4_5ratio_1week\tuser_roomservice_4_4ratio_1week\tuser_roomservice_4_2ratio_1week\tuser_roomservice_4_3ratio_1week\tuser_roomservice_4_0ratio_1week\tuser_ordnum_1month\tuser_avgprice_1month\tuser_medprice_1month\tuser_minprice_1month\tuser_maxprice_1month\tuser_roomservice_3_123ratio_1month\tuser_roomservice_7_1ratio_1month\tuser_roomservice_7_0ratio_1month\tuser_roomservice_4_5ratio_1month\tuser_roomservice_4_4ratio_1month\tuser_roomservice_4_2ratio_1month\tuser_roomservice_4_3ratio_1month\tuser_roomservice_4_0ratio_1month\tuser_ordnum_3month\tuser_avgprice_3month\tuser_medprice_3month\tuser_minprice_3month\tuser_maxprice_3month\tuser_roomservice_3_123ratio_3month\tuser_roomservice_7_1ratio_3month\tuser_roomservice_7_0ratio_3month\tuser_roomservice_4_5ratio_3month\tuser_roomservice_4_4ratio_3month\tuser_roomservice_4_2ratio_3month\tuser_roomservice_4_3ratio_3month\tuser_roomservice_4_0ratio_3month\tbasic_week_ordernum_ratio\tbasic_recent3_ordernum_ratio\tbasic_comment_ratio\tbasic_30days_ordnumratio\tbasic_30days_realratio\troom_30days_ordnumratio\troom_30days_realratio\torderid_lastord\torderdate_lastord\thotelid_lastord\troomid_lastord\tbasicroomid_lastord\trank_lastord\treturn_lastord\tprice_last_lastord\troomservice_2_lastord\troomservice_3_lastord\troomservice_4_lastord\troomservice_5_lastord\troomservice_6_lastord\troomservice_8_lastord\troomtag_2_lastord\troomtag_3_lastord\troomtag_4_lastord\troomtag_5_lastord\troomtag_6_lastord\tstar_lastord\thotel_minprice_lastord\tbasic_minprice_lastord".split("\t")
    //val schema = StructType(schemaString.map(fieldName => StructField(fieldName, StringType, true)))
    val schema = StructType(Array(StructField("orderid", StringType, true),StructField("uid", StringType, true),StructField("orderdate", StringType, true),StructField("hotelid", StringType, true),StructField("basicroomid", StringType, true),StructField("roomid", StringType, true),StructField("orderlabel", StringType, true),StructField("star", StringType, true),StructField("rank", StringType, true),StructField("returnvalue", StringType, true),StructField("price_deduct", StringType, true),StructField("basic_minarea", StringType, true),StructField("basic_maxarea", StringType, true),StructField("roomservice_1", StringType, true),StructField("roomservice_2", StringType, true),StructField("roomservice_3", StringType, true),StructField("roomservice_4", StringType, true),StructField("roomservice_5", StringType, true),StructField("roomservice_6", StringType, true),StructField("roomservice_7", StringType, true),StructField("roomservice_8", StringType, true),StructField("roomtag_1", StringType, true),StructField("roomtag_2", StringType, true),StructField("roomtag_3", StringType, true),StructField("roomtag_4", StringType, true),StructField("roomtag_5", StringType, true),StructField("roomtag_6", StringType, true),StructField("user_confirmtime", StringType, true),StructField("user_avgadvanceddate", StringType, true),StructField("user_avgstar", StringType, true),StructField("user_avggoldstar", StringType, true),StructField("user_avgrecommendlevel", StringType, true),StructField("user_avgroomnum", StringType, true),StructField("ordertype_1_ratio", StringType, true),StructField("ordertype_2_ratio", StringType, true),StructField("ordertype_3_ratio", StringType, true),StructField("ordertype_4_ratio", StringType, true),StructField("ordertype_5_ratio", StringType, true),StructField("ordertype_6_ratio", StringType, true),StructField("ordertype_7_ratio", StringType, true),StructField("ordertype_8_ratio", StringType, true),StructField("ordertype_9_ratio", StringType, true),StructField("ordertype_10_ratio", StringType, true),StructField("ordertype_11_ratio", StringType, true),StructField("user_avgdealpriceholiday", StringType, true),StructField("user_avgdealpriceworkday", StringType, true),StructField("user_avgdealprice", StringType, true),StructField("user_avgpromotion", StringType, true),StructField("user_avgprice_star", StringType, true),StructField("orderbehavior_1_ratio", StringType, true),StructField("orderbehavior_2_ratio", StringType, true),StructField("orderbehavior_3_ratio_1week", StringType, true),StructField("orderbehavior_4_ratio_1week", StringType, true),StructField("orderbehavior_5_ratio_1week", StringType, true),StructField("orderbehavior_3_ratio_1month", StringType, true),StructField("orderbehavior_4_ratio_1month", StringType, true),StructField("orderbehavior_5_ratio_1month", StringType, true),StructField("orderbehavior_3_ratio_3month", StringType, true),StructField("orderbehavior_4_ratio_3month", StringType, true),StructField("orderbehavior_5_ratio_3month", StringType, true),StructField("orderbehavior_6_ratio", StringType, true),StructField("orderbehavior_7_ratio", StringType, true),StructField("orderbehavior_8", StringType, true),StructField("orderbehavior_9", StringType, true),StructField("user_ordernum", StringType, true),StructField("user_activation", StringType, true),StructField("user_avgprice", StringType, true),StructField("user_maxprice", StringType, true),StructField("user_minprice", StringType, true),StructField("user_stdprice", StringType, true),StructField("user_cvprice", StringType, true),StructField("user_citynum", StringType, true),StructField("user_avgroomarea", StringType, true),StructField("user_roomservice_4_0ratio", StringType, true),StructField("user_roomservice_4_2ratio", StringType, true),StructField("user_roomservice_4_3ratio", StringType, true),StructField("user_roomservice_4_4ratio", StringType, true),StructField("user_roomservice_4_1ratio", StringType, true),StructField("user_roomservice_4_5ratio", StringType, true),StructField("user_roomservice_3_123ratio", StringType, true),StructField("user_roomservice_6_2ratio", StringType, true),StructField("user_roomservice_6_1ratio", StringType, true),StructField("user_roomservice_6_0ratio", StringType, true),StructField("user_roomservice_5_1ratio", StringType, true),StructField("user_roomservice_7_0ratio", StringType, true),StructField("user_roomservice_2_1ratio", StringType, true),StructField("user_roomservice_8_1ratio", StringType, true),StructField("user_rank_ratio", StringType, true),StructField("user_roomservice_5_345ratio", StringType, true),StructField("user_ordnum_1week", StringType, true),StructField("user_avgprice_1week", StringType, true),StructField("user_medprice_1week", StringType, true),StructField("user_minprice_1week", StringType, true),StructField("user_maxprice_1week", StringType, true),StructField("user_roomservice_3_123ratio_1week", StringType, true),StructField("user_roomservice_7_1ratio_1week", StringType, true),StructField("user_roomservice_7_0ratio_1week", StringType, true),StructField("user_roomservice_4_5ratio_1week", StringType, true),StructField("user_roomservice_4_4ratio_1week", StringType, true),StructField("user_roomservice_4_2ratio_1week", StringType, true),StructField("user_roomservice_4_3ratio_1week", StringType, true),StructField("user_roomservice_4_0ratio_1week", StringType, true),StructField("user_ordnum_1month", StringType, true),StructField("user_avgprice_1month", StringType, true),StructField("user_medprice_1month", StringType, true),StructField("user_minprice_1month", StringType, true),StructField("user_maxprice_1month", StringType, true),StructField("user_roomservice_3_123ratio_1month", StringType, true),StructField("user_roomservice_7_1ratio_1month", StringType, true),StructField("user_roomservice_7_0ratio_1month", StringType, true),StructField("user_roomservice_4_5ratio_1month", StringType, true),StructField("user_roomservice_4_4ratio_1month", StringType, true),StructField("user_roomservice_4_2ratio_1month", StringType, true),StructField("user_roomservice_4_3ratio_1month", StringType, true),StructField("user_roomservice_4_0ratio_1month", StringType, true),StructField("user_ordnum_3month", StringType, true),StructField("user_avgprice_3month", StringType, true),StructField("user_medprice_3month", StringType, true),StructField("user_minprice_3month", StringType, true),StructField("user_maxprice_3month", StringType, true),StructField("user_roomservice_3_123ratio_3month", StringType, true),StructField("user_roomservice_7_1ratio_3month", StringType, true),StructField("user_roomservice_7_0ratio_3month", StringType, true),StructField("user_roomservice_4_5ratio_3month", StringType, true),StructField("user_roomservice_4_4ratio_3month", StringType, true),StructField("user_roomservice_4_2ratio_3month", StringType, true),StructField("user_roomservice_4_3ratio_3month", StringType, true),StructField("user_roomservice_4_0ratio_3month", StringType, true),StructField("basic_week_ordernum_ratio", StringType, true),StructField("basic_recent3_ordernum_ratio", StringType, true),StructField("basic_comment_ratio", StringType, true),StructField("basic_30days_ordnumratio", StringType, true),StructField("basic_30days_realratio", StringType, true),StructField("room_30days_ordnumratio", StringType, true),StructField("room_30days_realratio", StringType, true),StructField("orderid_lastord", StringType, true),StructField("orderdate_lastord", StringType, true),StructField("hotelid_lastord", StringType, true),StructField("roomid_lastord", StringType, true),StructField("basicroomid_lastord", StringType, true),StructField("rank_lastord", StringType, true),StructField("return_lastord", StringType, true),StructField("price_last_lastord", StringType, true),StructField("roomservice_2_lastord", StringType, true),StructField("roomservice_3_lastord", StringType, true),StructField("roomservice_4_lastord", StringType, true),StructField("roomservice_5_lastord", StringType, true),StructField("roomservice_6_lastord", StringType, true),StructField("roomservice_8_lastord", StringType, true),StructField("roomtag_2_lastord", StringType, true),StructField("roomtag_3_lastord", StringType, true),StructField("roomtag_4_lastord", StringType, true),StructField("roomtag_5_lastord", StringType, true),StructField("roomtag_6_lastord", StringType, true),StructField("star_lastord", StringType, true),StructField("hotel_minprice_lastord", StringType, true),StructField("basic_minprice_lastord", StringType, true)))
    //然后将schema应用到Rdd上
    val df = sqlContext.createDataFrame(rowRdd, schema)
    df.registerTempTable("tbl_train_data")
    sqlContext.sql("select * from tbl_train_data limit 10").foreach(print)
    println("lines count:"+lines.count()+"etl count:"+rowRdd.count())
    rowRdd.saveAsTextFile("F:\\Study\\ml\\match\\ctrip\\ctrip\\etl_competition_train.txt")
  }
}
